utils.py: add parallel make helpers
PATCH 1/1 utils.py: add parallel make helpers
The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.
The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.
Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.
--- meta/classes/go.bbclass | 19 +--------------- meta/classes/waf.bbclass | 24 +------------------- meta/lib/oe/utils.py | 43 ++++++++++++++++++++++++++++++++++++ meta/recipes-core/ovmf/ovmf_git.bb | 2 +- meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)
meta/classes/go.bbclass +1 -18
@@ -1,23 +1,6 @@ inherit goarch ptest -def get_go_parallel_make(d): - pm = (d.getVar('PARALLEL_MAKE') or '').split() - # look for '-j' and throw other options (e.g. '-l') away - # because they might have a different meaning in golang - while pm: - opt = pm.pop(0) - if opt == '-j': - v = pm.pop(0) - elif opt.startswith('-j'): - v = opt[2:].strip() - else: - continue - - return '-p %d' % int(v) - - return "" - -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}" GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
meta/classes/waf.bbclass +1 -23
@@ -3,28 +3,6 @@ DISABLE_STATIC = "" EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" -def get_waf_parallel_make(d): - pm = d.getVar('PARALLEL_MAKE') - if pm: - # look for '-j' and throw other options (e.g. '-l') away - # because they might have different meaning in bjam - pm = pm.split() - while pm: - v = None - opt = pm.pop(0) - if opt == '-j': - v = pm.pop(0) - elif opt.startswith('-j'): - v = opt[2:].strip() - else: - v = None - - if v: - v = min(64, int(v)) - return '-j' + str(v) - - return "" - python waf_preconfigure() { import subprocess from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() { do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" waf_do_compile() { - ${S}/waf build ${@get_waf_parallel_make(d)} + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)} } waf_do_install() {
meta/lib/oe/utils.py +43 -0
49 lines not on this page · raw message
meta/recipes-core/ovmf/ovmf_git.bb +1 -1
@@ -151,7 +151,7 @@ do_compile_class-native() { do_compile_class-target() { export LFLAGS="${LDFLAGS}" - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}" OVMF_ARCH="X64" if [ "${TARGET_ARCH}" != "x86_64" ] ; then OVMF_ARCH="IA32"
meta/recipes-support/boost/boost.inc +1 -23
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \ # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j # https://svn.boost.org/trac/boost/ticket/7634 -def get_boost_parallel_make(d): - pm = d.getVar('PARALLEL_MAKE') - if pm: - # look for '-j' and throw other options (e.g. '-l') away - # because they might have different meaning in bjam - pm = pm.split() - while pm: - v = None - opt = pm.pop(0) - if opt == '-j': - v = pm.pop(0) - elif opt.startswith('-j'): - v = opt[2:].strip() - else: - v = None - - if v: - v = min(64, int(v)) - return '-j' + str(v) - - return "" - -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}" BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \ ${BJAM_TOOLS} \ -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
Signature · 1 line
2.14.3
+ v = max(limit, v)
this seems to be doing the opposite of what the limit was used for before:
- v = min(64, int(v))
Quoted 219 lines · Joshua Watt
The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.
The first helper (
parallel_make()) extracts the integer number of parallel build threads fromPARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/go.bbclass | 19 +--------------- meta/classes/waf.bbclass | 24 +------------------- meta/lib/oe/utils.py | 43 ++++++++++++++++++++++++++++++ ++++++ meta/recipes-core/ovmf/ovmf_git.bb | 2 +- meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)meta/classes/go.bbclass +1
-18@@ -1,23 +1,6 @@ inherit goarch ptest -def get_go_parallel_make(d): - pm = (d.getVar('PARALLEL_MAKE') or '').split() - # look for '-j' and throw other options (e.g. '-l') away - # because they might have a different meaning in golang - while pm: - opt = pm.pop(0) - if opt == '-j': - v = pm.pop(0) - elif opt.startswith('-j'): - v = opt[2:].strip() - else: - continue - - return '-p %d' % int(v) - - return "" - -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}" GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"meta/classes/waf.bbclass +1
-23@@ -3,28 +3,6 @@ DISABLE_STATIC = "" EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" -def get_waf_parallel_make(d): - pm = d.getVar('PARALLEL_MAKE') - if pm: - # look for '-j' and throw other options (e.g. '-l') away - # because they might have different meaning in bjam - pm = pm.split() - while pm: - v = None - opt = pm.pop(0) - if opt == '-j': - v = pm.pop(0) - elif opt.startswith('-j'): - v = opt[2:].strip() - else: - v = None - - if v: - v = min(64, int(v)) - return '-j' + str(v) - - return "" - python waf_preconfigure() { import subprocess from distutils.version import StrictVersion@@ -47,7 +25,7 @@ waf_do_configure() { do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" waf_do_compile() { - ${S}/waf build ${@get_waf_parallel_make(d)} + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)} }
waf_do_install(){meta/lib/oe/utils.py +0
-0@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1",falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue,
falsevalue, d)meta/lib/oe/utils.py +3
-0+def parallel_make(d): + """ + Return the integer value for the number of parallel threads to usewhen
+ building, scraped out ofPARALLEL_MAKE. If no parallelization option
ismeta/lib/oe/utils.py +22
-0+ found, returns None + + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer. + """ + pm = (d.getVar('PARALLEL_MAKE') or '').split() + # look for '-j' and throw other options (e.g. '-l') away + while pm: + opt = pm.pop(0) + if opt == '-j': + v = pm.pop(0) + elif opt.startswith('-j'): + v = opt[2:].strip() + else: + continue + + return int(v) + + return None + +def parallel_make_argument(d, fmt, limit=None): + """ + Helper utility to construct a parallel make argument from the numberof
meta/lib/oe/utils.py +3
-0+ parallel threads specified in PARALLEL_MAKE. + + Returns the input format string `fmt` where a single '%d' will beexpanded
+ with the number of parallel threads to use. Iflimitis specified,
the
+ number of parallel threads will be no larger than it. If no
parallelizationmeta/lib/oe/utils.py +3
-0+ option is found in PARALLEL_MAKE, returns an empty string + + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d")will return
meta/lib/oe/utils.py +9
-0+ "-n 10" + """ + v = parallel_make(d) + if v: + if limit: + v = max(limit, v) + return fmt % v + return '' + def packages_filter_out_system(d): """ Return a list of packages from PACKAGES with the "system" packagessuch as
meta/recipes-core/ovmf/ renamed from meta/recipes-core/ovmf/ovmf_git.bb +0
-0ovmf_git.bb
indexfa0d66291d1..8750b3c528d100644meta/recipes-core/ovmf/ovmf_git.bb +1
-1@@ -151,7 +151,7 @@ do_compile_class-native() { do_compile_class-target() { export LFLAGS="${LDFLAGS}" - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}" OVMF_ARCH="X64" if [ "${TARGET_ARCH}" != "x86_64" ] ; then OVMF_ARCH="IA32"meta/recipes-support/boost/boost.inc +0
-0@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \ # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greaterparallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634meta/recipes-support/boost/boost.inc +1
-23-def get_boost_parallel_make(d): - pm = d.getVar('PARALLEL_MAKE') - if pm: - # look for '-j' and throw other options (e.g. '-l') away - # because they might have different meaning in bjam - pm = pm.split() - while pm: - v = None - opt = pm.pop(0) - if opt == '-j': - v = pm.pop(0) - elif opt.startswith('-j'): - v = opt[2:].strip() - else: - v = None - - if v: - v = min(64, int(v)) - return '-j' + str(v) - - return "" - -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d',limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \ ${BJAM_TOOLS} \ -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \--
2.14.3--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Quoted 9 lines · Martin Jansa
+ v =
max(limit, v)this seems to be doing the opposite of what the limit was used for before:
- v =
min(64, int(v))
Hmm, you're right
On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected]> wrote:
The code to extract the integer number of parallel build threads and
construct a new argument from them has started to be copied in multiple
locations, so create two new helper utilities to aid recipes.
The first helper (parallel_make()) extracts the integer number of
parallel build threads from PARALLEL_MAKE. The second
(parallel_make_argument()) does the same and then puts the result back
into a format string, optionally clamping it to some maximum value.
Additionally, rework the oe-core recipes that were manually doing this
to use the new helper utilities.
---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py | 43 ++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb | 2 +-
meta/recipes-support/boost/boost.inc | 24 +-------------------
5 files changed, 47 insertions(+), 65 deletions(-)
meta/classes/go.bbclass +0 -0
index 09b01a84c37..7ecd8c92541 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest
-def get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE') or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
meta/classes/waf.bbclass +0 -0
index bdbdc56767c..f9a851d0759 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""
EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
-def get_waf_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
python waf_preconfigure() {
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {
do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
waf_do_compile() {
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
}
waf_do_install() {
meta/lib/oe/utils.py +0 -0
index 7a79d752b69..ec91927233a 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
+def parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to use when
+ building, scraped out of PARALLEL_MAKE. If no parallelization option is
+ found, returns None
+
+ e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
+ """
+ pm = (d.getVar('PARALLEL_MAKE') or '').split()
+ # look for '-j' and throw other options (e.g. '-l') away
+ while pm:
+ opt = pm.pop(0)
+ if opt == '-j':
+ v = pm.pop(0)
+ elif opt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+def parallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the number of
+ parallel threads specified in PARALLEL_MAKE.
+
+ Returns the input format string fmt where a single '%d' will be expanded
+ with the number of parallel threads to use. If limit is specified, the
+ number of parallel threads will be no larger than it. If no parallelization
+ option is found in PARALLEL_MAKE, returns an empty string
+
+ e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
+ "-n 10"
+ """
+ v = parallel_make(d)
+ if v:
+ if limit:
+ v = max(limit, v)
+ return fmt % v
+ return ''
+
def packages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system" packages such as
meta/recipes-core/ovmf/ovmf_git.bb +0 -0
index fa0d66291d1..8750b3c528d 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb
+++ b/meta/recipes-core/ovmf/ovmf_git.bb
@@ -151,7 +151,7 @@ do_compile_class-native() {
do_compile_class-target() {
export LFLAGS="${LDFLAGS}"- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"meta/recipes-support/boost/boost.inc +0 -0
index 41fc90fb211..0461ec6fcf1 100644
--- a/meta/recipes-support/boost/boost.inc
+++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
# use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
-def get_boost_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \--
2.14.3
--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Quoted 6 lines · Martin Jansa
+ v =
max(limit, v)this seems to be doing the opposite of what the limit was used for before:
- v =
min(64, int(v))
moreover, I think the limit should be based on ncpu() instead of hard coding it.
Quoted 227 lines · On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected]…
The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.
The first helper (
parallel_make()) extracts the integer number of parallel build threads fromPARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.
Signed-off-by Joshua Watt <[email protected]<mailto:[email protected]>>
---
meta/classes/go.bbclass | 19 +--------------- meta/classes/waf.bbclass | 24 +------------------- meta/lib/oe/utils.py | 43
++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +- meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index
09b01a84c37..7ecd8c92541100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest-def
get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE')or '').split()
- # look for '-j' and throw other options (e.g. '-l') away - # because they might have a different meaning in golang - while pm:
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass indexbdbdc56767c..f9a851d0759100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@DISABLE_STATIC= ""EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
-def
get_waf_parallel_make(d):
- pm =d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away - # because they might have different meaning in bjam - pm =pm.split()
- while pm:
- v = None
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v =min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
pythonwaf_preconfigure(){
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@waf_do_configure(){do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" waf_do_compile() {
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
}waf_do_install() {
diff --gita/meta/lib/oe/utils.pyb/meta/lib/oe/utils.pyindex7a79d752b69..ec91927233a100644
---a/meta/lib/oe/utils.py
+++b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ defany_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features,
truevalue, falsevalue, d)+def
parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to use when
+ building, scraped out ofPARALLEL_MAKE. If no parallelization option is
+ found, returns None
+
+ e.g. ifPARALLEL_MAKE= "-j 10", this will return 10 as an integer. + """
+ pm = (d.getVar('PARALLEL_MAKE')or '').split()
+ # look for '-j' and throw other options (e.g. '-l') away + while pm:
+ opt =pm.pop(0)
+ if opt == '-j':
+ v =pm.pop(0)
+ elifopt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+defparallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the number of
+ parallel threads specified inPARALLEL_MAKE.
+
+ Returns the input format stringfmtwhere a single '%d' will be expanded
+ with the number of parallel threads to use. Iflimitis specified, the
+ number of parallel threads will be no larger than it. If no parallelization
+ option is found inPARALLEL_MAKE, returns an empty string +
+ e.g. ifPARALLEL_MAKE= "-j 10",parallel_make_argument(d, "-n %d")will return
+ "-n 10"
+ """
+ v =parallel_make(d)
+ if v:
+ if limit:
+ v =max(limit, v)
+ return fmt % v
+ return ''
+
defpackages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system"
packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
indexfa0d66291d1..8750b3c528d100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
+++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
@@ -151,7 +151,7 @@ do_compile_class-native() {do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc
index41fc90fb211..0461ec6fcf1100644
--- a/meta/recipes-support/boost/boost.inc
+++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \# use
PARALLEL_MAKEto speed up the build, but limit it by -j 64,
greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
<https://svn.boost.org/trac/boost/ticket/7634>
-defget_boost_parallel_make(d):
- pm =d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away - # because they might have different meaning in bjam - pm =pm.split()
- while pm:
- v = None
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v =min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
--
2.14.3--
_______________________________________________
Openembedded-core mailing list [email protected] <mailto:[email protected]>
http://lists.openembedded.org/mailman/listinfo/openembedded-core
<http://lists.openembedded.org/mailman/listinfo/openembedded-core>
On Feb 17, 2018 18:25, "Khem Raj" <[email protected]> wrote:
Quoted 7 lines · Martin Jansa
+ v =
max(limit, v)this seems to be doing the opposite of what the limit was used for before:
- v =
min(64, int(v))
moreover, I think the limit should be based on ncpu() instead of hard coding it.
I don't know about that. Doesn't PARALLEL_MAKE already default to something based on ncpu()? My understanding of the limit was that some recipes (boost? waf?) have a limit on the amount of parallism they can reasonably stand, and you don't want those recipes to break if someone sets PARALLEL_MAKE="-j
1000".
As long as that is true I think this is correct. Whether those recipes still need to limit their parallism is another discussion.
On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected] <mailto:
Quoted 233 lines
[email protected]>> wrote:
The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.
The first helper (
parallel_make()) extracts the integer number of parallel build threads fromPARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.
Signed-off-by Joshua Watt <[email protected]<mailto:[email protected]>>
---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py| 43
++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)
diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index
09b01a84c37..7ecd8c92541100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest-def
get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE')or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p
%d')}"GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass indexbdbdc56767c..f9a851d0759100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@DISABLE_STATIC= ""
EXTRA_OECONF_append= " ${PACKAGECONFIG_CONFARGS}"-def
get_waf_parallel_make(d):
- pm =d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm =pm.split()
- while pm:
- v = None
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v =min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
pythonwaf_preconfigure(){
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@waf_do_configure(){do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
waf_do_compile(){
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
limit=64)}
}
waf_do_install(){
diff --gita/meta/lib/oe/utils.pyb/meta/lib/oe/utils.pyindex7a79d752b69..ec91927233a100644
---a/meta/lib/oe/utils.py
+++b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ defany_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features,
truevalue, falsevalue, d)+def
parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to
use when
+ building, scraped out ofPARALLEL_MAKE. If no parallelization
option is
+ found, returns None
+
+ e.g. ifPARALLEL_MAKE= "-j 10", this will return 10 as an
integer.+ """ + pm = (d.getVar('PARALLEL_MAKE') or '').split() + # look for '-j' and throw other options (e.g. '-l') away + while pm:+ opt =
pm.pop(0)
+ if opt == '-j':
+ v =pm.pop(0)
+ elifopt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+defparallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the
number of
+ parallel threads specified inPARALLEL_MAKE.
+
+ Returns the input format stringfmtwhere a single '%d' will
be expanded
+ with the number of parallel threads to use. Iflimitis
specified, the
+ number of parallel threads will be no larger than it. If no
parallelization
+ option is found inPARALLEL_MAKE, returns an empty string
+
+ e.g. ifPARALLEL_MAKE= "-j 10", parallel_make_argument(d, "-n
%d") will return
+ "-n 10"
+ """
+ v =parallel_make(d)
+ if v:
+ if limit:
+ v =max(limit, v)
+ return fmt % v
+ return ''
+
defpackages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system"
packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
indexfa0d66291d1..8750b3c528d100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
+++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>@@ -151,7 +151,7 @@ do_compile_class-native() {
do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc index41fc90fb211..0461ec6fcf1100644
--- a/meta/recipes-support/boost/boost.inc +++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@BJAM_TOOLS= "--ignore-site-config \# use
PARALLEL_MAKEto speed up the build, but limit it by -j 64,
greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
<https://svn.boost.org/trac/boost/ticket/7634>
-defget_boost_parallel_make(d):
- pm =d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm =pm.split()
- while pm:
- v = None
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v =min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS= '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam\ -- 2.14.3 -- _______________________________________________ Openembedded-core mailing list [email protected] <mailto:[email protected]> http://lists.openembedded.org/mailman/listinfo/openembedded-core <http://lists.openembedded.org/mailman/listinfo/openembedded-core>
Quoted 1 line
--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Quoted 19 lines · Joshua Watt
On Feb 17, 2018 18:25, "Khem Raj" <[email protected]> wrote:
Quoted 7 lines · Martin Jansa
+ v =
max(limit, v)this seems to be doing the opposite of what the limit was used for before:
- v =
min(64, int(v))moreover, I think the limit should be based on
ncpu()instead of hard coding it.
Quoted 6 lines
I don't know about that. Doesn't
PARALLEL_MAKEalready default to
something based onncpu()? My understanding of the limit was that some
recipes (boost? waf?) have a limit on the amount of parallism they can reasonably stand, and you don't want those recipes to break if someone sets PARALLEL_MAKE="-j 1000".
Ah yes that’s a fair point
Quoted 249 lines
As long as that is true I think this is correct. Whether those recipes still need to limit their parallism is another discussion.
On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected]
Quoted 234 lines
<mailto:[email protected]>> wrote:
The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in
multiple
locations, so create two new helper utilities to aid recipes.The first helper (
parallel_make()) extracts the integer number of parallel build threads fromPARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.
Signed-off-by Joshua Watt <[email protected]<mailto:[email protected]>>
---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py| 43
++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)
diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index
09b01a84c37..7ecd8c92541100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest-def
get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE')or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p
%d')}"GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass indexbdbdc56767c..f9a851d0759100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@DISABLE_STATIC= ""
EXTRA_OECONF_append= " ${PACKAGECONFIG_CONFARGS}"-def
get_waf_parallel_make(d):
- pm =d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm =pm.split()
- while pm:
- v = None
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v =min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
pythonwaf_preconfigure(){
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@waf_do_configure(){do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
waf_do_compile(){
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
limit=64)}
}
waf_do_install(){
diff --gita/meta/lib/oe/utils.pyb/meta/lib/oe/utils.pyindex7a79d752b69..ec91927233a100644
---a/meta/lib/oe/utils.py
+++b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ defany_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features,
truevalue, falsevalue, d)+def
parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to
use when
+ building, scraped out ofPARALLEL_MAKE. If no parallelization
option is
+ found, returns None
+
+ e.g. ifPARALLEL_MAKE= "-j 10", this will return 10 as an
integer.+ """ + pm = (d.getVar('PARALLEL_MAKE') or '').split() + # look for '-j' and throw other options (e.g. '-l') away + while pm:+ opt =
pm.pop(0)
+ if opt == '-j':
+ v =pm.pop(0)
+ elifopt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+defparallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the
number of
+ parallel threads specified inPARALLEL_MAKE.
+
+ Returns the input format stringfmtwhere a single '%d' will
be expanded
+ with the number of parallel threads to use. Iflimitis
specified, the
+ number of parallel threads will be no larger than it. If no
parallelization
+ option is found inPARALLEL_MAKE, returns an empty string
+
+ e.g. ifPARALLEL_MAKE= "-j 10", parallel_make_argument(d, "-n
%d") will return
+ "-n 10"
+ """
+ v =parallel_make(d)
+ if v:
+ if limit:
+ v =max(limit, v)
+ return fmt % v
+ return ''
+
defpackages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system"
packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
indexfa0d66291d1..8750b3c528d100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
+++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>@@ -151,7 +151,7 @@ do_compile_class-native() {
do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc index41fc90fb211..0461ec6fcf1100644
--- a/meta/recipes-support/boost/boost.inc +++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@BJAM_TOOLS= "--ignore-site-config \# use
PARALLEL_MAKEto speed up the build, but limit it by -j 64,
greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
<https://svn.boost.org/trac/boost/ticket/7634>
-defget_boost_parallel_make(d):
- pm =d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm =pm.split()
- while pm:
- v = None
- opt =pm.pop(0)
- if opt == '-j':
- v =pm.pop(0)
- elifopt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v =min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS= '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam\ -- 2.14.3 -- _______________________________________________ Openembedded-core mailing list [email protected] <mailto:[email protected]> http://lists.openembedded.org/mailman/listinfo/openembedded-core <http://lists.openembedded.org/mailman/listinfo/openembedded-core>Quoted 1 line
--
_______________________________________________
Openembedded-core mailing list
[email protected]http://lists.openembedded.org/mailman/listinfo/openembedded-core